/*
 * SmartCam master processor library
 * User front end.
 */

#include <stdarg.h>
#include <getopt.h>
#include <log.h>

#include "globals.h"
#include "settings.h"
#include "management.h"
#include "dispatch.h"
#include "results.h"
#include "threads.h"
#include "mapping.h"
#include "trace.h"

#include "scsim.h"

log_default_streams_t scl;
sc_settings_t settings;
tmr_t totaltime;
int SC_DEFAULT_BUFFER, SC_FRAME_BUFFER;
int affinity=-1;

double last_makespan_prediction=0, last_energy_prediction=0;

extern scos_mutex_t thr_mutex;
extern scos_cond_t  thr_cond;
extern sc_thread_t *thr_thread;

extern scos_mutex_t scop_finished_mutex; 
extern scos_cond_t scop_finished_cond; 

/* Initialize the system */
void scInit(int argc, char **argv)
{
  char optchar;

#ifndef LIBDYNAMIC
  liblog_init();
  _scuInit();
#endif

  logRegisterThread("main");  

  scl = logGetDefaultStreams();
  logEdit(scl.info, LOG_GROUP_SET_LEVEL);

  lprintf(STREAM(scl.notice), "Initializing SmartCam system");

  SC_DEFAULT_BUFFER = 64*1024;
  SC_FRAME_BUFFER = 1024*1024;

  /* Read parameters */
  memset(&settings, 0, sizeof(sc_settings_t));
  settings.argc = argc;
  settings.argv = argv;
  settings.multiprocessing = 1;

  opterr = 0;
  while ((optchar = getopt(argc, argv, "cn:m:o:t:w:a:s:T:O:r:phN:1234R")) != -1)
  {
    switch (optchar)
    {
      case 'c':
        settings.master = 1;
        break;
      case 'n':
        settings.name = optarg;
        break;
      case 'm':
        settings.machinesfile = optarg;
        break;
      case 'o':
        settings.benchfile = optarg;
        break;
      case 'a':
        settings.avfile = optarg;
        break;
      case 't':
        settings.tracefile = optarg;
        break;
      case 'w':
	settings.writetodisk = atoi(optarg);
        break;
      case 's':
	settings.simulate = optarg;
        break;
      case 'T':
	settings.intracefile = optarg;
	break;
      case 'O':
	settings.outbenchfile = optarg;
	break;
      case 'r':
	settings.reportfile = optarg;
	break;
      case 'p':
	settings.spawn = 1;
	break;
      case 'h':
        settings.hardaffinity = 1;
        break;
      case 'N':
        settings.benchmarknoise = atof(optarg);
        break;
      case '1':
        settings.multiprocessing = 1;
        break;
      case '2':
        settings.multiprocessing = 2;
        break;
      case '3':
        settings.multiprocessing = 3;
        break;
      case '4':
        settings.multiprocessing = 4;
        break;
      case 'R':
        settings.noresources = 1;
    }
  }

  scSetInit();

  scManInit();

  if (!settings.master)
  {
    lprintf(STREAM(scl.notice), "Awaiting commands...");
    while (1)
      usleep(100000L);
  }

  scThrInit();
  scMapInit();
  //scAvlInit();
  scDisInit();
  scResInit();
#ifndef PSOS
  scTraInit();
#endif

  //scSetPrint(stdout);

  printf("%% SmartCam system initialized\n");

  totaltime = tmrInit();
  tmrStart(&totaltime);
}

void *_scOp(int line, int parent, char *format, int op, ...)
{
  va_list ap;
  int id, id2, id3;
  table_t args;
  sc_argdecode_t *d;
  sc_arg_t *a, *a2;
  sc_op_t *o;
  sc_var_t *v;
  sc_instance_t *i;
  sc_origin_t *ro=NULL;
  sc_originref_t *r;
  scs_operation_t *ao;
  char *buf;
  int *retval=NULL;
  int hasOrigin = 0;

  scManAcquireLock();

//  abortOnNegative(STREAM(scl.err), id = tbladdnew(&sc_ops, &o));
  if ((id = tbladdnew(&sc_ops, &o)) < 0)
  {
    printf("here!\n");
    exit(-1);
  }

  o->id = id;
  o->type = op;
  o->line = line;
  o->parent = parent;
  o->parent_condition = scTraGetCondition(o->parent);
  
  o->affinity = affinity;
  affinity = -1;

  if (tblget(&scs_operations, o->type, &ao) && ao->kernel)
    o->text = ao->kernel;
  else
    o->text = "unknown";

  o->format = format; /* WARNING: ASSUMES STATIC STRING */
  abortOnNegative(STREAM(scl.err), tblcreate(&o->arguments, sizeof(sc_arg_t)));

  lprintf(STREAM(scl.info), "Enqueueing operation %s(\"%s\") on line %d at id %d",
	  o->text, format, line, o->id);

  /* Decode arguments */
  va_start(ap, op);
  args = scManArgDecode(format, ap);
  va_end(ap);
  
  /* Build argument list */
  id = -1;
  while ((id = tblgetnexti(&args, id)) >= 0)
  {
    abortOnNULL(STREAM(scl.err), tblget(&args, id, &d));
    abortOnNegative(STREAM(scl.err), tbladdnew(&o->arguments, &a));

    a->id = id;
    a->intent = d->intent;
    a->operation = o;
  }

  /* We resolve inputs before outputs because of possible aliasing */

  /* Resolve inputs */
  id = -1;
  while ((id = tblgetnexti(&args, id)) >= 0)
  {
    abortOnNULL(STREAM(scl.err), tblget(&args, id, &d));

    if (d->intent == SC_INTENT_IN)
    {
      abortOnNULL(STREAM(scl.err), tblget(&o->arguments, id, &a));
      i = scManFindInstance(d->address);

      if (i)
      {
	/* Not immediate */
	v = i->variable;
	lprintf(STREAM(scl.debug), "Input argument %d of op %d is non-immediate %d type %d",
		id, o->id, v->id, v->type);

	abortOnError(STREAM(scl.err), v->type != d->type);
	a->instance = i;
	abortOnNegative(STREAM(scl.err), tbladd(&v->consumers, a));

	if (!i->multiuse)
	{
	  /* This is a temporary variable, the result of a function call */
	  lprintf(STREAM(scl.debug), "Implicit finalization of single-use variable");
	  _finalize(d->address);
	  
	  if (i->temporary)
	  {
	    lprintf(STREAM(scl.crawl), "Freeing temporary variable");
	    free(d->address);
	  }
	}
      }
      else
      {
	/* Make a variable structure and copy data, because it might be gone before we start
	 * mapping. Note that we do not add a hash: this variable cannot be referenced
	 * by the user. */
	v = scManAddVariable(d->type, d->size);
	i = scManAddInstance(v, NULL);
	i->thread = scManGetThread();

	lprintf(STREAM(scl.debug),
		"Input argument %d of op %d is immediate type %d assigned to new id %d",
		id, o->id, d->type, v->id);

	abortOnError(STREAM(scl.err), d->type == SC_VAR_STREAM);

	abortOnNULL(STREAM(scl.err), v->data = malloc(v->size));
	memcpy(v->data, d->address, v->size);
	v->valid = 1;
	i->finalized = 1;

	v->producer = NULL;
	a->instance = i;
	abortOnNegative(STREAM(scl.err), tbladd(&v->consumers, a));	

	/* Remove temporary, if this is one */
	scManCheckDestroyTemporary(d->address);
      }
    }
  }
  
  /* Resolve outputs */
  id = -1;
  while ((id = tblgetnexti(&args, id)) >= 0)
  {
    abortOnNULL(STREAM(scl.err), tblget(&args, id, &d));

    if (d->intent == SC_INTENT_OUT)
    {
      if (!d->address)
      {
	lprintf(STREAM(scl.debug),
		"Creating temporary variable for output argument %d", id);

	/* Make a temporary for this variable, and return it as the
	 * result of the call */
	abortOnError(STREAM(scl.err), retval != NULL);
	retval = d->address = (int*) malloc(sizeof(int));
      }

      abortOnNULL(STREAM(scl.err), tblget(&o->arguments, id, &a));
      i = scManFindInstance(d->address);

      if (i && !i->virtualproducer)
      {
	/* We've already got a producer for this variable, or are not
	 * responsible anymore. In either case, finalize and overwrite. */
	_finalize(d->address);
	i = NULL;
      }

      if (!i)
      {
	/* Create a new variable */
	v = scManAddVariable(d->type, d->size);
	i = scManAddInstance(v, d->address);
	i->thread = scManGetThread();

	if (retval == d->address)
	{
	  /* Mark the freeing of this variable once it is used */
	  i->temporary = 1;
	  i->multiuse = 0;
	}

	buf = "new";
      }
      else
      {
	/* Reify producership */
	i->virtualproducer = 0;
	v = i->variable;
	buf = "existing";

	/* Get virtual origin */
	if (tblgetelements(&v->originrefs) != 1)	  
	  abortErrorl((STREAM(scl.err), "Variable with virtual producer can only have one origin"));
	abortOnNULL(STREAM(scl.err), tblget(&v->originrefs, tblgetnexti(&v->originrefs, -1), &r));
	ro = r->origin;

	/* Remove our own reference to it */
	abortOnNegative(STREAM(scl.err), tbldel(&r->origin->references, r->id, 1));
	abortOnError(STREAM(scl.err), tblclear(&v->originrefs, 0));
      }

      a->instance = i;
      i->variable->producer = a;

      lprintf(STREAM(scl.debug),
	      "Output argument %d of op %d is type %d assigned to %s id %d",
	      id, o->id, d->type, buf, v->id);

      /* Set origin(s) */
      id2 = -1;
      hasOrigin = 0;
      while (tblgetnextelement(&o->arguments, &id2, &a2))
      {
	if (a2->intent == SC_INTENT_IN)
	{
	  /* state = MAX(target_operation_argument->state) */
	  int state = 0;
	  if (ao)
	  {
	    scs_target_operation_t *ato;
	    scs_target_operation_argument_t *atoa;

	    id3 = -1;
	    while (tblgetnextelement(&ao->targets, &id3, &ato))
	      if (tblget(&ato->arguments, a2->id, &atoa))
		state = MAX(state, atoa->state * atoa->elementsize * atoa->chunksize);
	  }

	  id3 = -1;
	  while (tblgetnextelement(&a2->instance->variable->originrefs, &id3, &r))
	  {
	    scManCheckReferenceOrigin(v, r, state);
	    hasOrigin = 1;
	  }
	}
      }
      
      if (!hasOrigin)
      {
	/* No origin. Create a new one. */
	scManCreateOrigin(v, 0);
      }

      /* If we just reified a producer, replace all references to the (virtual)
       * original origin to (actual) new the origin(s) of the variable */
      if (ro)
	scManReplaceOrigin(ro, &v->originrefs);
    }
  }

  tbldestroy(&args, 1);

  if (!settings.noresources && scMapOperation(o, 0))
  {
    scManShedLock();
    _schedule(0);
  }
  else
    scManShedLock();

//  scDisOperation(o);

//scManDump();

  return retval;
}

#ifdef __cplusplus
void _scThread(char *text, char *format, void (*op)(...), ...)
#else
void _scThread(char *text, char *format, void (*op)(), ...)
#endif
{
  va_list ap;
  int id;
  sc_cusparam_t *p;
  sc_argdecode_t *d;
  sc_var_t *v;
  sc_instance_t *i, *newi;
  void *temp;
  scos_thread_t thread;
  char *buf;

  scManAcquireLock();

  lprintf(STREAM(scl.info), "Starting custom thread %s(\"%s\")", text, format);

  abortOnNULL(STREAM(scl.err), p = (sc_cusparam_t*) malloc(sizeof(sc_cusparam_t)));
  p->text = text;
  p->op = op;

  /* Decode arguments */
  va_start(ap, op);
  p->args = scManArgDecode(format, ap);
  va_end(ap);

  /* Start thread. It will wait with starting the process because it needs to
   * acquire the scMan lock first */
  scos_mutex_lock(&thr_mutex);
  scos_thread_create(&thread, scThrRun, p);
  scos_cond_wait(&thr_cond, &thr_mutex);
  scos_mutex_unlock(&thr_mutex);

  /* We resolve inputs before outputs because of possible aliasing */

  /* Resolve inputs */
  id = -1;
  while ((id = tblgetnexti(&p->args, id)) >= 0)
  {
    abortOnNULL(STREAM(scl.err), tblget(&p->args, id, &d));

    if (d->intent == SC_INTENT_IN)
    {
      i = scManFindInstance(d->address);

      if (i)
      {
	/* Not immediate */
	v = i->variable;
	abortOnError(STREAM(scl.err), v->type != d->type);

	/* Create an instance for reference by the new thread */
	newi = scManAddInstance(v, malloc(v->size));
	newi->thread = thr_thread;

	lprintf(STREAM(scl.debug),
		"Input argument %d of %s is non-immediate %d type %d",
		id, text, v->id, v->type);

	/* Pass it to the new thread */
	d->address = newi->address;

	if (!i->multiuse)
	{
	  /* This is a temporary variable, the result of a function call */
	  lprintf(STREAM(scl.debug), "Implicit finalization of single-use variable");
	  _finalize(i->address);
	  
	  if (i->temporary)
	  {
	    lprintf(STREAM(scl.crawl), "Freeing temporary variable");
	    free(i->address);
	  }
	}
      }
      else
      {
	/* Immediate. Cannot be a stream */
	lprintf(STREAM(scl.debug),
		"Input argument %d of %s is immediate type %d", id, text, d->type);

	abortOnError(STREAM(scl.err), d->type == SC_VAR_STREAM);

	/* Pass a copy of the immediate variable to new thread */
	temp = d->address;
	abortOnNULL(STREAM(scl.err), d->address = malloc(d->size));
	memcpy(d->address, temp, d->size);
      }
    }
  }
  
  /* Resolve outputs */
  id = -1;
  while ((id = tblgetnexti(&p->args, id)) >= 0)
  {
    abortOnNULL(STREAM(scl.err), tblget(&p->args, id, &d));

    if (d->intent == SC_INTENT_OUT)
    {
      i = scManFindInstance(d->address);

      if (i && !i->virtualproducer)
      {
	/* We've already got a producer for this variable, or are not
	 * responsible anymore. In either case, finalize and overwrite. */
	_finalize(d->address);
	i = NULL;
      }

      if (!i)
      {
	/* Create a new variable and an instance for the OLD thread */
	v = scManAddVariable(d->type, d->size);
	i = scManAddInstance(v, d->address);
	i->thread = scManGetThread();
	
	buf = "new";
      }
      else
      {
	/* Pass virtual producership to new instance */
	i->virtualproducer = 0;
	v = i->variable;
	buf = "existing";
      }

      /* Create a new instance for the new thread */
      newi = scManAddInstance(v, malloc(v->size));
      newi->thread = thr_thread;
      newi->virtualproducer = 1;
      
      /* Create a (virtual) origin */
      scManCreateOrigin(v, 1);
      
      lprintf(STREAM(scl.debug),
	      "Output argument %d of %s is type %d assigned to %s id %d",
	      id, text, d->type, buf, v->id);

      /* Pass it to the new thread */
      d->address = newi->address;
    }
  }

  scManShedLock();
  /* New thread will destroy argument table */
}

void _finalize(void *s)
{
  sc_instance_t *i;
  sc_var_t *v;
  int unblock=0;

  scManAcquireLock();

  if (!(i = scManFindInstance(s)))
  {
    scManShedLock();
    return;
  }

  v = i->variable;

  lprintf(STREAM(scl.debug), "Finalizing variable %d", v->id);

  if (!i->blocked)
  {
    i->blocked = 1;
    unblock = 1;
  }

  i->finalized = 1;
  abortOnNegative(STREAM(scl.err), tbldelhash(&sc_vars, &s));
  abortOnNegative(STREAM(scl.err), tbldelhash(&v->instances, &s));

  scDisVariable(v);

  if (unblock)
    i->blocked = 0;

  scManCheckDestroyInstance(i);

  scManShedLock();
}

/* DO NOT CALL FROM WITHIN A GLOBAL LOCK! */
void _block(void *address)
{
  sc_instance_t *i;
  sc_var_t *v;
  sc_thread_t *t;
  int id=-1;

  scManAcquireLock();

  if (!(i = scManFindInstance(address)))
  {
    scManShedLock();
    return;
  }

  v = i->variable;

  /* Prevent destruction while blocking */
  i->blocked = 1;

  /* This instance won't be referenced anymore, so finalize */
  _finalize(address);

  t = scManGetThread();
  t->blocked = 1;
  t->instance = i;

  /* 
   * Now, dispatch an output buffer for all non-dispatched stream variables that are
   * blocked or not finalized in at least one blocked thread, and share an origin
   * with any variable currently being blocked upon.
   */

  /* More down to earth: schedule if there are no running threads */
  while (tblgetnextelement(&sc_threads, &id, &t))
    if (!t->blocked)
      break;

  scManShedLock();

  if (id < 0)
    _schedule(1);
  
  /* Wait for instance to become immediate */
  if (v->type == SC_VAR_STREAM)
  {
    lprintf(STREAM(scl.info), "Waiting for variable %d to be destroyed", v->id);

    scos_mutex_lock(&v->lock);
    while (!scManIsVariableDestroyed(v)) scos_cond_wait(&v->condition, &v->lock);
    scos_mutex_unlock(&v->lock);
  }
  else
  {
    lprintf(STREAM(scl.info), "Waiting for non-immediate %d to become immediate", v->id);

    scos_mutex_lock(&v->lock);
    while (!v->valid) scos_cond_wait(&v->condition, &v->lock);
    scos_mutex_unlock(&v->lock);
  }

  lprintf(STREAM(scl.debug), "Done waiting for variable %d", v->id);

  scManAcquireLock();

  i->blocked = 0;

  scManCheckDestroyInstance(i);

  scManShedLock();
}

int *scint(int i)
{
  int *address;

  address = (int*) scManAddTemporary(sizeof(int));
  *address = i;

  return address;
}

double *scdbl(double f)
{
  double *address;

  address = (double*) scManAddTemporary(sizeof(double));
  *address = f;

  return address;
}

int scExit(void)
{
  int id=-1, id2=-1;
  sc_var_t *v;
  sc_instance_t *i;
  tmr_t timer = tmrInit();

  scManAcquireLock();

  /* Finalize all variables */
  while (tblgetnextelement(&sc_vars, &id, &v))
    while (tblgetnextelement(&v->instances, &id2, &i))
      if (!i->finalized)
	_finalize(i->address);

  scManShedLock();

  /* Schedule unscheduled operations */
  /* Note: no cycles should be necessary, as all variables
   * have just been finalized */
  _schedule(0);

  tmrStart(&timer);

  _awaitall();

  tmrStop(&totaltime);
  tmrStop(&timer);

  if (settings.simulate)
    scSimExit(settings.reportfile);
  else if (settings.reportfile)
  {
    FILE *f = fopen(settings.reportfile, "w");
    fprintf(f, "<document>\n");
    fprintf(f, "  <!-- WARNING: time is time spent in scExit only -->\n");
    fprintf(f, "  <E time=\"%f\"/>\n", tmrGetValueInUs(&timer)/1000000.0);
    fprintf(f, "  <E predicted-time=\"%f\"/>\n", last_makespan_prediction);
    fprintf(f, "</document>\n");
    fclose(f);
  }

  //scos_usleep(1000000L);

#ifndef PSOS
  scTraWrite();
#endif

  printf("%% Exiting SmartCam system after %f seconds (%f seconds in scExit)\n",
	 tmrGetValueInUs(&totaltime)/1000000.0,
	 tmrGetValueInUs(&timer)/1000000.0);

  return 0;
}

int scTrace(int type, int line, int parent, int condition)
{
  scTraUpdateCondition(type, line, parent, condition);
  
  return condition;
}

/* DO NOT CALL FROM WITHIN A GLOBAL LOCK! */
void _schedule(int make_it_run)
{
  int id=-1, all_mapped=0;
  sc_op_t *so;
  sc_thread_t *t;
  sc_var_t *v;

  scManAcquireLock();

  t = scManGetThread();

  while (!all_mapped)
  {
    lprintf(STREAM(scl.info), "Mapping unmapped operations");

    all_mapped = 1;

    while (tblget(&sc_ops, id = tblgetnexti(&sc_ops, id), &so))
      if (!so->processor)
	all_mapped &= !scMapOperation(so, 1);

    if (!all_mapped || make_it_run)
    {
      lprintf(STREAM(scl.info), "Inserting cycles for all unfinalized variables");

      while (tblgetnextelement(&sc_vars, &id, &v))
	if (v->type == SC_VAR_STREAM && v->producer && v->producer->operation->dispatched &&
	    v->producer->buffer < SC_FRAME_BUFFER && !scManIsVariableFinalized(v))
	  all_mapped &= !scMapOperation(scManInsertLoop(v), 1);
    }

    if (!all_mapped)
    {
      /* Wait for an operation to finish and try again */
      lprintf(STREAM(scl.info), "Mapping failed; waiting for an operation to finish");

      if (t) t->blocked = 1;

      scos_mutex_lock(&scop_finished_mutex);

      scManShedLock();

      scos_cond_wait(&scop_finished_cond, &scop_finished_mutex);
      scos_mutex_unlock(&scop_finished_mutex);

      scManAcquireLock();

      if (t) t->blocked = 0;
    }
  }

  lprintf(STREAM(scl.info), "Mapping finished");

  scManShedLock();
}

void _cycle(void *address)
{
  sc_instance_t *i;
  sc_var_t *v;

  scManAcquireLock();

  if (!(i = scManFindInstance(address)))
  {
    scManShedLock();
    return;
  }

  v = i->variable;

  scMapOperation(scManInsertLoop(v), 1);

  scManShedLock();
}

void _multi(void *address)
{
  sc_instance_t *i;

  scManAcquireLock();

  if (!(i = scManFindInstance(address)))
  {
    scManShedLock();
    return;
  }

  i->multiuse = 1;

  scManShedLock();
}

void _allmulti(int multiuse)
{
  settings.multiuse = multiuse;
}

void _mapto(int _affinity)
{
  affinity = _affinity;
}

void _setbuffersize(int size)
{
  int id=-1;
  scs_target_t *t;
  sc_proc_t *p;
  
  while (tblgetnextelement(&scs_targets, &id, &t))
  {
    t->buffer = size;
    if (tblget(&sc_procs, id, &p))
      p->bufsize = size;
  }
}

void _awaitall(void)
{
  int id=-1, finished;
  sc_op_t *o;
  char buf[PATH_MAX];

  printf("%% Waiting for all operations to finish\n");

  /* Wait for all operations to finish */
  do
  {
    finished = 1;
    *buf = 0;

    scos_mutex_lock(&scop_finished_mutex);

    while (tblget(&sc_ops, id = tblgetnexti(&sc_ops, id), &o))
    {
      finished *= o->finished;
      
      if (!o->finished)
	sprintf(&buf[strlen(buf)], "%d ", o->id);
    }

    if (!finished)
    {
      lprintf(STREAM(scl.crawl), "Waiting for operations [ %s]", buf);

      scos_cond_wait(&scop_finished_cond, &scop_finished_mutex);
      scos_mutex_unlock(&scop_finished_mutex);
    }
  } while (!finished);

  scos_mutex_unlock(&scop_finished_mutex);
}
